package crawler.web.framework;

import java.beans.PropertyDescriptor;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.LinkedList;
import java.util.List;

import javax.annotation.Resource;

import library.Tracer;
import library.database.mongo.MongoDB;
import library.functions.Function3PV;
import library.thread.flow.ParameterBox;
import library.thread.flow.WorkflowContainer;
import library.thread.flow.WorkflowNode;
import library.utils.bean.MyBeanUtils;

import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import crawler.web.beans.CrawlPageContext;
import crawler.web.beans.PageFingerPrint;
import crawler.web.deduplicate.IKUtil;
import crawler.web.deduplicate.SimHashUtil;
import crawler.web.engine.config.SpBN;
import crawler.web.engine.config.Suite;
import crawler.web.engine.custom.MongoDBAdapter;
import crawler.web.utils.FingerPrintDPL;

/**
 * 从context1取出数据, 存入context2, 并去重
 * 
 * @author waynechen Feb 8, 2012 11:18:01 PM
 */
@Component("AnalyzerDedu")
@Scope(value = "prototype")
public class AnalyzerDedu extends WorkflowNode {
	@Autowired(required = true)
	public AnalyzerDedu(//
	@Value("#{T(crawler.web.framework.CrawlerContainer).getNO()}") String name, //
			@Qualifier("CrawlerContainer") WorkflowContainer<Crawler> container//
	) {
		super("AnalyzerDedu " + name, container);
	}

	private int waitTimes = 0;

	@Resource(name = SpBN.bean_MongoDB)
	private MongoDB mongo;

	@Resource(name = "MongoDBAdapter")
	private MongoDBAdapter dbAdapter;

	@Resource(name = SpBN.conf_parseTabName1)
	private String tableContext1;

	@Resource(name = SpBN.conf_parseTabName2)
	private String tableContext2;

	@Resource(name = SpBN.conf_table_empty_wait_time)
	private Integer waitTime;

	@Resource(name = SpBN.bean_fingerprint_berkeleydb)
	private FingerPrintDPL simpleDPL;

	@Override
	protected void nodeExecute(ParameterBox args) {
		Tracer.info("线程: " + name + " 开始");

		try {
			while (true) {
				if (isStopped())
					break;

				//处理消息
				handleMessage();

				//从tableContext1取
				CrawlPageContext tar = dbAdapter.getNext(//
				tableContext1, Suite.ParseStatus.PARSED, Suite.ParseStatus.DEDUPLICATED);

				//无数据等待
				if (tar == null) {
					Tracer.info("ac3: 分析库1中没有等待分析的数据,等待中...");
					sleepWithoutThrowing(10000);
					if (waitTimes++ > (waitTime / 10000))
						getContainer().sendMessage2me(this, WorkflowContainer.MSG_STOP_ALL_SUBNODES, null);
					continue;
				}
				waitTimes = 0;

				//从tableContext2查出数据
				CrawlPageContext src = null;

				//如果是景点
				if (tar.getPageType() == Suite.PageType.SIGHT.getExpr()) {
					src = mongo.findOne(//
					tableContext2, //
							String.format("{'location.identity':'%s'}", tar.getLocation().getIdentity()),//
							CrawlPageContext.class);
				}

				//數據庫中已經存在此條記錄
				//将从tableContext2中查出的, 覆盖掉tableContext1
				if (src != null) {
					MyBeanUtils.copyBean(src, tar, new Function3PV<PropertyDescriptor, Object, Object>() {
						@Override
						public void apply(PropertyDescriptor p, Object src, Object tar) {
							try {
								if (p.getPropertyType().getName().contains("String")) {
									String ret = p.getReadMethod().invoke(src) + "";
									if (StringUtils.isNotEmpty(ret))
										cp(p, src, tar);
								} else if (p.getPropertyType().getName().endsWith("Integer")) {
									Integer ret = (Integer) p.getReadMethod().invoke(src);
									if (!ret.equals(-1))
										cp(p, src, tar);
								} else if (p.getPropertyType().getName().endsWith("Double")) {
									Double ret = (Double) p.getReadMethod().invoke(src);
									if (!ret.equals(0.0))
										cp(p, src, tar);
								} else {
									cp(p, src, tar);
								}
							} catch (Exception e) {
								Tracer.exception(e);
							}
						}

						public void cp(PropertyDescriptor p, Object src, Object tar) throws IllegalAccessException, InvocationTargetException {
							p.getWriteMethod().invoke(tar, p.getReadMethod().invoke(src));
						}
					});

					//更新到tableContext2
					CrawlPageContext query = new CrawlPageContext();
					query.set_id(src.get_id());
					mongo.update(tableContext2, query, tar);
				}
				//數據庫中沒有此記錄
				else {
					//计算语义指纹
					String keywords = tar.getKeywords();
					String text = getText(tar);
					List<String> list = new LinkedList<String>();
					list.add(tar.getTitle().toString());
					tar.setFingerPrint(getFingerPrint(keywords, text, list));

					//存入berkeleydb 本地内存磁盘加速去重
					PageFingerPrint pp = new PageFingerPrint();
					pp.set_id(tar.get_id());
					pp.setFingerPrint(tar.getFingerPrint());
					simpleDPL.save(pp);

					//插入到tableContext2
					mongo.save(tableContext2, tar);
				}
			}
		} catch (Exception e) {
			Tracer.exception("线程: " + name + " 发生了异常");
			Tracer.exception(e);
		} finally {
			Tracer.info("线程: " + name + " 结束");
		}
	}

	/**
	 * 如果没有 textbuffer则找最大的n个tag节点
	 * 
	 * @author waynechen
	 * @param pageContext
	 * @return
	 */
	public String getText(CrawlPageContext pageContext) {
		return pageContext.getSight().getTextBuffer().toString();
	}

	/**
	 * 语义指纹 排重<br>
	 * 使用之前, 要将文档进行同义词替换, 以提高排重的正确率<br>
	 * 
	 * @author waynechen
	 * @param name
	 * @param content
	 * @return
	 * @throws IOException
	 */
	private static long getFingerPrint(String name, String content, List<String> additionalkeyList) throws IOException {
		int min = Math.min(content.length(), 20000);
		content = content.substring(0, min);

		//1取得关键词 从name与content中
		List<String> words = IKUtil.getWords(name);
		words.addAll(additionalkeyList);
		List<String> list = IKUtil.getFirst$kWords(words, name, 3);

		words = IKUtil.getWords(content);
		words.addAll(additionalkeyList);
		list.addAll(IKUtil.getFirst$kWords(words, content, 5));

		//2给simHashUtil计算出simHash值
		long simHash = SimHashUtil.getSimHash(list);
		return simHash;
	}

	@Override
	protected void handleMessage() {
	}
}

//
//
//
//
//
//
//
//
//
//
